import java.util.ArrayList;
import java.util.Random;

public class GridWorld {

	private State[][] states;
	private int[] wind;
	private final float STOCHASTIC_VAR = 1 / 3f;
	private final int STANDARD_REWARD = -1;
	private final int GOAL_REWARD = 10;
	private State currentState;
	private State goalState;
	private Random random;
	private final float ALFA;
	private final float GAMMA;
	private final Coordinate START;
	private int nbOfSteps;

	private ArrayList<Integer> Steps = new ArrayList<Integer>();
	private ArrayList<Integer> collReward = new ArrayList<Integer>();

	public GridWorld(int x, int y, int[] wind, Coordinate start,
			Coordinate goal, float alfa, float gamma) {
		// if(x != wind.length)
		// throw new
		// IllegalArgumentException("The wind-vector must have the same size as the gridworld's x-dimension.");
		// this.wind = wind;
		initGridWorld(x, y);
		this.START = new Coordinate(start.getX(), start.getY());
		this.currentState = this.getState(start);
		this.goalState = this.getState(goal);
		this.random = new Random();
		this.ALFA = alfa;
		this.GAMMA = gamma;
		this.nbOfSteps = 0;
	}

	private State getState(Coordinate coordinate) {
//		int x, y;
		if (coordinate.getX() < 0)
			// x = 0;
			return null;
		else if (coordinate.getX() > 6)
			// x = 11;
			return null;
		// else
		// x = coordinate.getX();

		if (coordinate.getY() < 0)
			// y = 0;
			return null;
		else if (coordinate.getY() > 6)
			// y = 6;
			return null;
		// else
		// y = coordinate.getY();

		// return this.states[x][y];
		return this.states[coordinate.getX()][coordinate.getY()];
	}

	private void initGridWorld(int x, int y) {
		this.states = new State[x][y];
		for (int i = 0; i < x; i++) {
			for (int j = 0; j < y; j++) {
				this.states[i][j] = new State(i, j);
			}
		}
		this.states[0][6] = null;
		this.states[0][3] = null;
		this.states[0][0] = null;
		this.states[1][3] = null;
		this.states[3][6] = null;
		this.states[3][5] = null;
		this.states[3][3] = null;
		this.states[3][5] = null;
		this.states[3][6] = null;
		this.states[5][3] = null;
		this.states[6][0] = null;
		this.states[6][3] = null;
		this.states[6][6] = null;
	}

	public String toString() {
		String result = "***Q-Values*** \n";
		for (int i = this.states[0].length - 1; i >= 0; i--) {
			for (int j = 0; j < this.states.length; j++) {
				result = result.concat(this.states[j][i] + " ");
			}
			result = result.concat("\n");
		}
		// result = result.concat("\n ***Wind*** \n");
		// for(Integer integer : wind){
		// result = result.concat(integer + " ");
		// }
		return result;
	}

	protected void takeStep(Policy policy) {
		// Kies action volgens epsilon-greedy policy
		Direction direction = policy.getPreferredDirection(this
				.getCurrentState().getQValues());

		// Hou rekening met de wind
		// int wind = this.getWindDisplacement();
		int x = this.getCurrentState().getX() + direction.getXDiscplacement();
		int y = this.getCurrentState().getY() + direction.getYDisplacement()/*
																			 * +wind
																			 */;
		Coordinate coordinate = new Coordinate(x, y);

		// Hou rekening met rand van bord
		State newState = this.getState(coordinate);
		if (newState == null) {
			newState = this.currentState;
		}

		// Neem die action en sla reward en nieuwe state op
		int reward;
		if (newState.equals(goalState))
			reward = this.GOAL_REWARD;
		else
			reward = this.STANDARD_REWARD;

		// Bereken nieuwe Q-value
		float newQValue = this.calculateNewQValue(newState, direction, reward);
		this.getCurrentState().setQValue(direction, newQValue);

		// verwissel states
		this.setCurrentState(newState);
		nbOfSteps++;
	}

	private void setCurrentState(State state) {
		this.currentState = state;
	}

	public int getNbOfSteps() {
		return this.nbOfSteps;
	}

	public int getCollectedReward() {
		return (nbOfSteps * STANDARD_REWARD + GOAL_REWARD);
	}

	private float calculateNewQValue(State newState, Direction direction,
			int reward) {
		float result = this.getCurrentState().getQValue(direction);
		float maxQValue = this.getMaxQValue(newState);
		maxQValue *= this.GAMMA;
		maxQValue += reward;
		maxQValue -= this.getCurrentState().getQValue(direction);
		maxQValue *= this.ALFA;
		result += maxQValue;
		return result;
	}

	private float getMaxQValue(State newState) {
		float result = newState.getQValue(Direction.NORTH);
		for (Direction direction : Direction.values()) {
			if (newState.getQValue(direction) > result) {
				result = newState.getQValue(direction);
			}
		}
		return result;
	}

	private int getWindDisplacement() {
		int wind = this.getWindVector()[this.getCurrentState().getX()];
		float index = this.getRandom().nextFloat();
		if (index < this.STOCHASTIC_VAR) {
			// nr onder
			wind--;
		} else if (index < 2 * this.STOCHASTIC_VAR) {
			// nr boven
			wind++;
		} else {
			// doe niks
		}
		return wind;
	}

	private Random getRandom() {
		return this.random;
	}

	private int[] getWindVector() {
		return this.wind;
	}

	private State getCurrentState() {
		return this.currentState;
	}

	public boolean isInGoal() {
		return this.getCurrentState().equals(this.getGoalState());
	}

	private State getGoalState() {
		return this.goalState;
	}

	public void reset() {
		this.currentState = this.getState(START);
		Steps.add(nbOfSteps);
		collReward.add(this.getCollectedReward());
		nbOfSteps = 0;
	}

	public void resetBuffers() {
		Steps = new ArrayList<Integer>();
		collReward = new ArrayList<Integer>();
	}

	public ArrayList<Integer> getEpisodesCollectedReward() {
		return this.collReward;
	}

	public ArrayList<Integer> getEpisodesNbOfSteps() {
		return this.Steps;
	}

	public void writeDownInfo() {
		Utility.CreateFiles(Steps, collReward);
	}

	public String bestDirections() {
		String result = "***Directions*** \n";
		for (int i = this.states[0].length - 1; i >= 0; i--) {
			for (int j = 0; j < this.states.length; j++) {
				State state = this.states[j][i];
				if (state == null) {
					result = result.concat("(NULL)");
				} else {
					result = result.concat(this.states[j][i].bestDirection());
				}
			}
			result = result.concat("\n");
		}
		return result;
	}
}
